#!/usr/bin/python3
# -*- coding: utf-8 -*-

import requests
import json

from multiprocessing import Pool, freeze_support
import sys
import codecs
import logging
import os


base_url = 'https://www.engineeringvillage.com'
initial_url = base_url + '/search/submit.url'
next_url = base_url + '/search/results/quick.url'


def get_authors_from_conference(abbr, year, code):
    """ Return authors from a conference. """

    start_year = 1969
    end_year = 2018
    page_size = 25
    s=requests.Session()
    initial_payload = {
        'usageOrigin': 'searchform',
        'usageZone': 'quicksearch',
        'editSearch': '',
        'isFullJsonResult': 'true',
        'angularReq': 'true',
        'CID': 'searchSubmit',
        'searchtype': 'Quick',
        'origin': 'searchform',
        'category': 'quicksearch',
        'section1': 'CC',
        'searchWord1': code,
        'database': '1',
        'yearselect': 'yearrange',
        'startYear': str(start_year),
        'endYear': str(end_year),
        'updatesNo': '1',
        'language': 'NO-LIMIT',
        'doctype': 'NO-LIMIT',
        'sort': 'relevance',
        'treatmentType': 'NO-LIMIT',
        'pageSize': str(page_size)
    }
    
    r = s.get(initial_url, params=initial_payload)
    j = json.loads(r.text)
    search_id = j['searchMetaData']['searchId']
    results_count = j['pagenav']['resultscount']
    results_per_page = j['pagenav']['resultsperpage']

    author_list = []
    current_index = 1

    while current_index <= results_count:
        if current_index == 1:
            pass
        else:
            next_payload = {
                'navigator': 'NEXT',
                'SEARCHID': search_id,
                'database': '1',
                'angularReq': 'true',
                'isFullJsonResult': 'false',
                'usageOrigin': 'searchresults',
                'COUNT': str(current_index),
                'usageZone': 'nextpage'
            }
            #print("current_index:",current_index)
            #print("the current_index not equals 1")
            r = s.get(next_url, params=next_payload)
            j = json.loads(r.text)

        for result in j['results']:
            #assert(year == result['yr'])
            title = result['title']
            pages = result['pages']
            if result['authors'] is None:
                print('[Warning] skip "%s" without any author' % title, file=sys.stderr)
                continue
            for author_index, author in enumerate(result['authors']):
                author = result['authors'][author_index]
                name = author['name']
                affiliation = author['affils'][0]['name'] if author['affils'] is not None else ''
                country = ''
                email = author['email']
                author_item = [abbr, year, title, pages,
                        str(author_index+1), name, email, affiliation]
                author_list.append(author_item)
        current_index += results_per_page

    return author_list


def main():
    """ Crawl author lists from Engineering Village """
    if len(sys.argv) <= 1:
        print('Usage: EV.py <conference-list>', file=sys.stderr)
        return
    save_path='results-cc/'
    if os.path.exists(save_path):
        pass
    else:
        os.mkdir(save_path)
    filename = sys.argv[1]

    logging.basicConfig(level=logging.INFO)

    uopen = lambda path, mode: open(path, mode, encoding='utf-8')
    requests.session().cookies.clear()
    for line in uopen(filename, 'r'):
        abbr, year, code = line.rstrip().split('\t')
        print('[Working] extract %s%s papers with code %s' % (abbr, year, code), file=sys.stderr)
        author_list = get_authors_from_conference(abbr, year, code)
        if len(author_list) == 0:
            print('[Warning] %s%s papers not found' % (abbr, year), file=sys.stderr)
        # Output csv format: abbr, year, title, pages, order, author, email, affiliation
        csv_filename = '%s%s.csv' % (abbr, year)
        with open(save_path+csv_filename, 'wb') as csv:  # for excel (byte mode + BOM)
            csv.write(codecs.BOM_UTF8)
            add_quote = lambda x: '"%s"' % x if ',' in x else x
            csv.write('\n'.join(
                ','.join(add_quote(field) for field in author_info)
                for author_info in author_list).encode('utf-8'))
        print('[Done] %s written' % csv_filename, file=sys.stderr)
        print('', file=sys.stderr)


if __name__ == '__main__':
    freeze_support()
    main()
